package com.mofee.spider;

import com.mofee.db.DBUtils;
import com.mofee.entity.RegionEntity;
import com.mofee.util.HttpUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

/**
 * 全国省,市,区/县,乡/镇,村/社区 5级地址爬虫
 * 数据来源 国家统计局 http://www.stats.gov.cn
 *
 * @author Mofee.Zhang
 * @date 2018/4/24 17:13
 */
public class RegionSpider {
    private static String baseUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016";
    private static String[] levels = {"province", "city", "county", "town", "village"};

    public static void main(String[] args) {
        List<RegionEntity> regions = getProvince();

        if (regions.size() > 0) {

            insert2Db(regions);

            // 爬取省内地址列表
            regions.forEach(region -> getRegion(region.getId(), "/", 1));
        }
    }

    private static void getRegion(final Long parentId, final String parentUrl, final int level) {
        String url = baseUrl + parentUrl + parentId + ".html";
        System.out.println("url => " + url);
        String html = HttpUtils.get(url, "gbk", true);
        if (html != null) {
            Document document = Jsoup.parse(html);
            Elements trs = document.getElementsByClass(String.format("%str", levels[level]));
            // 遍历tr
            trs.forEach(tr -> {
                Elements tds = tr.getElementsByTag("td");
                Element firstTd = tds.first();
                Element lastTd = tds.last();
                String name = lastTd.text();
                Long id;
                Elements tagA = firstTd.getElementsByTag("a");
                boolean hasNext = false;
                if (tagA.size() > 0) {
                    String href = tagA.attr("href").replace(".html", "");
                    String[] strings = href.split("/");
                    id = Long.valueOf(strings[strings.length - 1]);
                    hasNext = true;
                } else {
                    id = Long.valueOf(firstTd.text());
                }
                RegionEntity region = new RegionEntity();
                region.setId(id);
                region.setName(name);
                region.setLevel(level);
                region.setParentId(parentId);

                System.out.println(region);

                insert2Db(region);

                if (hasNext) {
                    // 下钻爬取下层地址
                    String parentSid = parentId.toString();
                    getRegion(id, parentUrl + parentSid.substring(parentSid.length() - 2) + "/", level + 1);
                }
            });
        }
    }

    private static List<RegionEntity> getProvince() {
        String indexHtml = HttpUtils.get(baseUrl + "/index.html", "GBK");

        List<RegionEntity> regions = new LinkedList<>();

        // 爬取省列表
        if (indexHtml != null) {
            Document indexDocument = Jsoup.parse(indexHtml);
            Elements provinceTrs = indexDocument.getElementsByClass("provincetr");
            provinceTrs.forEach(tr -> tr.getElementsByTag("td").forEach(td -> {
                String name = td.text();
                Long id = null;
                Elements a = td.getElementsByTag("a");
                if (a != null && a.size() > 0) {
                    String href = a.attr("href");
                    if (href != null) {
                        id = Long.valueOf(href.replace(".html", ""));
                    }
                }
                RegionEntity region = new RegionEntity();
                region.setId(id);
                region.setName(name);
                region.setLevel(0);
                region.setParentId(0L);
                regions.add(region);
            }));
        }
        return regions;
    }

    private static void insert2Db(RegionEntity region) {
        DBUtils.exec("insert into sys_region(id, name, level, parent_id) values(?,?,?,?)",
                region.getId(), region.getName(), region.getLevel(), region.getParentId());
    }

    private static void insert2Db(List<RegionEntity> regions) {
        StringBuilder sql = new StringBuilder("insert into sys_region(id, name, level, parent_id) values");

        List<String> placeholders = new ArrayList<>(regions.size());

        List<Object> params = new ArrayList<>();

        regions.forEach(region -> {
            placeholders.add("(?,?,?,?)");

            params.add(region.getId());
            params.add(region.getName());
            params.add(region.getLevel());
            params.add(region.getParentId());
        });

        sql.append(StringUtils.join(placeholders.toArray(), ","));
        DBUtils.exec(sql.toString(), params.toArray());
    }
}
